import scrapy
from scrapy import Request, Selector

# from items import BookInfoItem
from booksSpider.items import BookInfoItem


class GetbookslinkSpider(scrapy.Spider):
    name = 'getBooksLink'
    allowed_domains = ['yuedu.163.com']
    target_url = "https://yuedu.163.com"

    # start_urls = ['http://yuedu.163.com/']
    def start_requests(self):
        # 分页爬取
        page_size = 20
        for page_num in range(1, 101):
            page_url = "{}/book/category/category/0_0_1/p{}/s{}".format(self.target_url, page_num, page_size)
            print("开始爬取书籍页面: {}".format(page_url))
            yield Request(url=page_url)

    def parse(self, response, **kwargs):
        link_list = Selector(text=response.text).xpath(
            "//div[@class='yd-book-content yd-book-content-standalone yd-store-content-container clearfix']/div/a/@href").extract()
        # 拿到链接列表之后进行第二次爬取，拿到具体的书籍数据
        for link in link_list:
            book_url = self.target_url + link
            print("开始爬取书籍信息：{}".format(book_url))
            yield Request(url=book_url, callback=self.book_info_parse)

    def book_info_parse(self, response, **kwargs):
        book_info = BookInfoItem()
        selector = Selector(text=response.text)

        book_info['book_name'] = selector.xpath("//div[@class='f-fl']/h3/@title").extract_first()
        book_info['book_author'] = selector.xpath("//div[@class='f-fl']/h3/span/a/text()").extract_first()
        book_info['book_abstract'] = selector.xpath(
            "//div[@class='f-fl']/div/text()").extract_first().strip()  # strip 去空格
        book_info['book_image'] = selector.xpath("//div[@class='m-bookdetail']/a/img/@src").extract_first()
        book_info['book_link'] = self.target_url + selector.xpath(
            "//div[@class='f-fl']/div[@class='ops']/a/@href").extract_first()

        book_state = selector.xpath("//div[@class='m-bookstatus']/table/tr/td[2]")
        book_info['book_type'] = book_state[2].xpath("//td/a/text()").extract_first()
        book_info['book_label'] = book_state[3].xpath("//td/div/a/text()").extract()
        book_info['book_score'] = selector.xpath(
            "//div[@class='starlevel']/span[@class='score']/text()").extract_first()

        yield book_info
